{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import time\n",
    "import random\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15000, 160)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../data/train_xy.csv')\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 159)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('../data/train_x.csv')\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cust_id</th>\n",
       "      <th>cust_group</th>\n",
       "      <th>y</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>...</th>\n",
       "      <th>x_148</th>\n",
       "      <th>x_149</th>\n",
       "      <th>x_150</th>\n",
       "      <th>x_151</th>\n",
       "      <th>x_152</th>\n",
       "      <th>x_153</th>\n",
       "      <th>x_154</th>\n",
       "      <th>x_155</th>\n",
       "      <th>x_156</th>\n",
       "      <th>x_157</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110000</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.354167</td>\n",
       "      <td>0.604988</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110001</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.012058</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>110002</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.565979</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>110003</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.316209</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110004</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.008061</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 160 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cust_id cust_group  y       x_1       x_2  x_3  x_4  x_5  x_6  x_7  ...    \\\n",
       "0   110000    group_3  0  0.354167  0.604988  -99  -99  -99  -99  -99  ...     \n",
       "1   110001    group_3  0  0.125000  0.012058  -99  -99  -99  -99  -99  ...     \n",
       "2   110002    group_3  0  0.333333  0.565979    0    0    0    0    0  ...     \n",
       "3   110003    group_3  0  0.208333  0.316209    0    0    0    0    1  ...     \n",
       "4   110004    group_3  0  0.208333  0.008061  -99  -99  -99  -99  -99  ...     \n",
       "\n",
       "   x_148  x_149  x_150  x_151  x_152  x_153  x_154  x_155  x_156  x_157  \n",
       "0      1      1      1      1      1      1      1      1      3    -99  \n",
       "1      1      1      1      1      1      1      1      1      2      2  \n",
       "2      1      1      2      1      1      1      1      1      2      2  \n",
       "3      2      1      1      1      1      1      1      1      2      4  \n",
       "4      1      1      1      1      1      1      1      1      2      1  \n",
       "\n",
       "[5 rows x 160 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x20cbe4c8c18>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAADuCAYAAAD7nKGzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAGE1JREFUeJzt3XmYFNW9xvHv6Z5hZlhlXwQtFAUV\nRRRjFBGuihgKlxi9KipkMYlGk3hjTOpeE4PJVSsxV2NyTW6iYlRc467lIxrAHRWXqGDYgqUi6rDO\nMDPM0t11/6hGWQamZ6a7T9Xp3+d55hkemO5+h2feOaeqT51SQRAghDBXQncAIURhScmFMJyUXAjD\nScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmF\nMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyUXAjDScmFMJyU\nXAjDScmFMFyZ7gCiOCzH6wkMA/oA5UCXHT5v/XMaqM1+1AAbgWrftWs1xBZ5oOT+5GawHK8LcAhw\nAGGZ98p+bP1zz06+RCNQDXwE/BNYsvXDd+01nXxuUUBS8hiyHE8B+wFfAo7Mfh4DVGiKtImw8O8B\nbwLzfdderimL2IGUPCYsxzsEsIFJwBFAb62B2vYhMA/4O/B337WrNecpWVLyiLIcrxw4HjiVsNzD\n9CbqlABYTFj4R4HnfdeWH7wikZJHiOV4SWAKcCZhuaM+WnfUB8Ac4HbftVfoDmM6KXkEWI43EPg2\n8F1gqOY4xfYKcAdwr+/aG3WHMZGUXCPL8cYDFwNfI3z7qpQ1AY8Dv/dd+wXdYUwiJS8yy/G6AucS\nlnuM5jhR9QpwHfCI79oZ3WHiTkpeJNn3sS8ErgAGaI4TF8uAq4G7fddO6w4TV1LyArMcLwGcB1wF\nWHrTxNZy4L+RsneIlLyALMebBlwDHKw7iyHeAS72XftF3UHiREpeAJbjHQX8BjhGdxZD3Qlc7rv2\nZ7qDxIGUPI8sx+sOuMD3AKU5julqgCuBm2QKv3tS8jyxHO944BbkuLvY3iacwr+kO0hUSck7KXsJ\n53XAd3RnKWEB8EfgMt+1m3SHiRopeSdYjjcFuJl4rys3ydvAWb5rL9MdJEqk5B1gOV4FcCPhMlQR\nLfWE0/fbdQeJCil5O1mOtzfwADBOdxaxW3cC3/Ndu053EN2k5O1gOd6JwN1AX91ZRE5WEE7f39Id\nRCfZyDFHluP9CHgSKXic7Ae8bDneabqD6CQjeRuya87/BHxTdxbRYRngEt+1/6Q7iA5S8t3Ivj32\nGDBRdxaRF9f6rv1fukMUm5R8FyzH2wOYS7hJojDH7cAFvmundAcpFil5KyzH6ws8A4zVnUUUxFzg\njFI58y4l34HleAMINxyUK8fM9jow2XftTbqDFJqcXd+G5XhDgOeQgpeCccBTluP10B2k0KTkWZbj\nDSMs+CjdWUTRHAk8YTlele4ghSTTdcByvD7AQmB/3VmEFnOBk33XbtEdpBBKfiTPvg/+EFLwUjYF\nuC17+ynjlHzJgb8g74OLcAfd63SHKISSLrnleFcAM3XnEJFxmeV4l+gOkW8le0xuOd5ZwD3INk1i\ney3ARN+1F+oOki8lWfLsRovzgUrdWUQkrQYO8117re4g+VBy0/XsYpeHkYKLXRsK3JXdMz/2jPgm\n2mk2MFB3CBF5k4Ff6A6RDyU1Xbcc7yLCDf+EyEUATPVd+yndQTqjZEpuOd5I4E2gq+4sIlbWEx6f\nf6g7SEeVxHTdcrxy4C6k4KL9+hKupYitkig54c0GD9cdQsTWFMvxztEdoqOMn65bjncM4YUnpfIL\nTRRGNTDKd+2NuoO0l9E/+Nlp+s0Y/n2KohhATJe9mv7Dfyly6ajIn29ajnes7hDtZex0PbsBxFLA\n+E0BRFEtBcb4rt2sO0iuTB7JXaTgIv9GAY7uEO1h5EhuOd5hhHt4ycUnohDqgOG+a6/THSQXpo7k\n1yEFF4XTHbhcd4hcGTeSW443FfB05xDGqwf28V27WneQtpg4kl+lO4AoCd2An+gOkQujRnLL8SYC\nz+rOIUrGFsLR/FPdQXbHtJH8x7oDiJJSRQzOtBszkluONwp4DznhJoqrEdjXd+01uoPsikkj+WVI\nwUXxVQKR3vzRiJHccryBwAdAhe4soiRVA8OiugrOlJH8EqTgQp8BwOm6Q+xK7EuevQPKRbpziJIX\n2Z/B2Jec8BY3fXWHECXvWMvxRugO0RoTSn627gBCZH1dd4DWxPrEW/aWs9WEa4mF0O0jwPJdO6M7\nyLbiPpJPQwouomMYMEl3iB3FveSx3VxPGOsU3QF2FNuSW47XE/iK7hxC7GCq7gA7im3JgdOQ+5mJ\n6NnPcrx9dYfYVpxLfrLuAELsQqRG8ziXPHa7ZoqSEamSx/IttOwVZ//UnUOIXWgE+viuvUV3EIjv\nSD5RdwAhdqMSOE53iK3iWnKZqouom6w7wFZSciEKIzI32IzdMbnleMOBVbpzCNGGzUAv37W1F0zr\nSK6UOkkptUwptVIpleteWXI8LuKgBxCJ98u1lVwplQRuIly1diBwjlLqwBweGplpkBBtOFR3ANA7\nkn8JWBkEwaogCJqBe4FTc3hcLr8IhIiCsboDgN6S70l4ad5Wq7N/1xYpuYiLkh/JW9tZdbcnKSzH\n2wMYVJg4QuRdyY/kqwmvv91qKNDW3tX7FS6OEHk32HK8PrpDlGl87UXAfkqp4cDHhNs4TW/jMcPz\n8cK1rz9K3dtzIYDuY6bQ84hT2fTiXdS9PZdE114A9D52BlX7HrHd44JUM5/e/VOCVAtkMnQdOZ49\nJpwLwJYP3mbTgtkE6Ra6DBpB36/8EJVIUr/sJWpeuItEVXf6n/4zklU9adn4CZuev4P+p/40H9+O\niLYhwAadAbSVPAiClFLqEmAukARmB0GwpI2H7dPZ121e61P39lwGzbgelSyn+v4rqdp3HAA9xp1G\nryN3s7NuspyBZ19DoksVQTrFp3f9hKp9DqfLkP1Z793AwLOvprzPnmx6YQ51786jx5gT2fzawww6\n/7fU//N56t97jp6Hn8ymF+5kjwnndfZbEfEwCFisM4DW98mDIHgyCIL9gyDYNwiCq3N4SKdH8pb1\nq6kYMopEeSUqkaRi2GgaVizM6bFKKRJdqgAIMinIpEEpMls2o5LllPcJzxtWWofSsPyl7IMSBOkW\nglQTKpGk8aPFJLv1/vxrhfG0n0OK27LWTv+Hdem3N40fLSa9pZZMSyNbVr1OunYdAJvffII1sy9h\n3ZO/I91Y1+rjg0yaNbd9n9V/OI9K61AqhowkUdWTIJOi6ZMVADQse+nz5+w1/hyq77+SRv8fdDtw\nIjUv30ev8bJrVQnRXnKdx+Qd0aOzT1Debxg9jzyD6vt+jiqvpMuA4ZBI0mPsVHodfTYoxaYX5rBx\n/i30m3rpTo9XiSRDvvEHMo11VD98Nc1rfbr0t+h/yk/YOP9mgnQLldZhkEgCUDV8LFXDw5Osde/O\no2rfcaTWr2bDaw+RqOxO7xO+Q6JcNrgxmPaSx20k73TJAXqMOZHBX7+RQef+mkRlD8p7DyHZrTcq\nkUSpBD3GTKH5k+W7fY5EZXcqhx3MllVvAlCx5wEMOvc3DJ5xA5XDDqK895Dtvj7T0kjd4nn0GGuz\n8fnb6Tv1UroMGkH9kmfz8S2J6JKSt1PPfDxJun4TAKnaahqWL6TrgRNJ1X1xArRh+ULK++298+Ma\nashkp/GZliYaP/gH5X2HbvecQaqF2lcfoPvY7feYrH31QXqOOwWVLCNoyd4XTyUIUk35+JZEdGkv\neclN1wHWPnINmS2bIZGkz+QLSVZ2Z90T/0PzZ6tAKcp6DaDPlPButKnN61n/1O8ZeOZVpOs2sM67\nAYIMBBm6jppA1xFfAqD2tYdoWPkaENDj0KlU7T3m89dLbV5P86cr2eOY8O22nl/6Kp/e+WMSld3o\nf/rP8vEtiegaoDtArC41tRxvM3IzBREvy3zXHqUzQGym65bjKaCb7hxCtJP22XKbJVdKXaKU6l2M\nMG3oTuvr3YWIMu0lzyXAIGCRUupNYDYwN9Azxy/X8Jol4yD1/sppyVfWlJPSHcUoKZKbwNaaIadj\ncqWUAk4EvgGMA+4Hbg2C4F+FjfcFy/EqgUhscWuqntTVnJl8/r2zkgsyI9THByYUUZjBxd2HzKrZ\n+a2aIsr5xJtSagxhyU8CFgBfBp4JguAnhYu3Pcvx0sToPEKcJcikJybeXjIj+fTGoxLvDa1ULZHY\nyiiGfGbV5OXCqo5qc7qulPoBMBNYB9wCXB4EQYtSKgGsAIpWcqABObteFBkSyQWZsYcsyISr9Yar\nNR/OSD7jT0u+0q0fNQcrRRfNEeNC+/FPLsfk/YDTgyD4YNu/DIIgo5SaVphYu1SPlFyL94Mhe12V\nmrnXVamZdGPL5tOTL755dnJ+6gD14ciECvoX6nXTmYBxN9ezZ48ET0zvutO/37+khVnPNqEUjBmY\n4O6vhV9z0px6Xlmd5pi9yrZ73LkPNfDuZxmm7V/GNceHy4l/9VwThwxMcOqogpz2qS3Ek7ZHmyUP\nguDK3fxbsW9VVF/k1xOtqKeqx53pyV++Mz0ZCIKjE0uWzEzOXTch8e7grqp5/3y+1o2vNnNAvwS1\nrSwMXLE+zbUvNvHSN7vRu0pRXZ/5/N8uP7qChpaAP7/R8vnfvfNZOvx8UXcm3FZPTWNAQ0vAa2vS\n/HxiRT5jb2t9oZ44V9pP77eTlDxylHo5M/qglzOjARiq1q45L/nMylOTL1UNYuPBSnX89tKrazN4\nK1JcMaGC6xc27/TvN7/ZwsVHdKF3VfjO6oBuX5yuOX6fMp71t58plydgSwtkgoDmdEAyAVcuaOKX\nkwpWcJCSt5uUPOJWB/2HuKnpQ9zUdKpoapiWXLjo3OS8xoPVqv2SKmjXOu5Ln2rkNydUsrm59ZPD\ny9eHI/f42fWkMzBrUgUnjdj1j/QB/ZPs1SvBYX+u5/xDylm5IUMAjB2cbE+s9tK6KwzEr+SbdAcQ\nudtCRde/pScd8bf0JAAOV8uWzix7+rPjEm/170bjAUrtenHTE8tbGNBNcfiQ5E4j8lapDKzYkOHZ\nmV1ZXRsw4bZ6Fn+vO3tU7nrN1O9O+mJicfI9Dfx5WiVXP9/E25+lmbxPGd8+PO/nE2Ukbye5PVKM\nvRGMHPVGy8hRAAPZUD29bN7y0xMvlg9Va0crtf2S5Zc+TPPYshRPrthMYwpqmwLOe2gLc06v+vxr\nhvZUfHlokvKkYnhvxch+CVasz3DEnm2PzI8ubWHc4CT1zQGL16a5/8yuHHtbPeceUk7X8rwurNQ+\nksftPeeiLb4RhfUZfQbckDrzmAnNNx45sun2sh80X/zGoszI51NBYjXAtSdUsvpHPfAv7cG9Z1Rx\n3PCy7QoOcNqochb44cm0dQ0Zlq/PsE/vtgvakg648dVmLh/fhYaWL9ZKZwJoTuf3+6TtHYgBUErN\nVkpVK6Xyvh9c3EZyKbmBmimveCwz/vDHmscDcLBateLrZXPXnJB4o09PGg5im8HoygWNjBuS5JSR\n5UzZN8nT/0px4E11JBNw3eRK+nYNv3TCbfUsXZehrjlg6PWbufWUKqZkj9dvWtTMzDHhiH3IwAQB\ncPCf6pg6omy3U/0OynX2+Vfgf4E78h0gbpeajgbe1Z1DFE8fatafnXx26ZnJ5xKW+vQgpfKzcUgR\n9WVWTU5TdqWUBTwRBMHofAaIW8m7ImfYS1YZqZbjE28tmZF8uuaIxFKri0prXROegw3Mqumb6xdL\nybMsx1sDDNadQ+g3Un34/ozk0x9+JbmoV282j1YqcoefLzOrZnyuX1yokkftPyUX/0JKLoBlwV7D\nr0hdMPyK1AXbXkEXjFAfHxCRK+iKvSK0VXEs+TLgGN0hRLTU0r3XrempR92anrr1Crp3InAFXSTO\nH8Wx5IuAb+kOIaJrN1fQde9HzegiXkH3Sq5fqJS6B5gE9FNKrQZ+EQTBrfkIEcdj8jHAP3TnEPHU\njS11X02+uPicwl9B1wT0ZFbNzovuiyyOJU8CNcimjqLTguCoxHvvfT05d+2ExDtD8nwF3UJm1Ryd\nx+frsNiVHMByvPnAv+nOIcyyJ2s/Ob/smRX5uIIOuJ5ZNZflLVwnxPGYHOA5pOQizz6m/2A3NX1w\nPq6gox3H44UW15I/qzuAMFtnrqDLeqngIXMU1+l6BeFlp3I7UFF0bV1BB7zDrJoxrT5Yg1iWHMBy\nvLmE20QLoU0XWpqmJBa9O6PsmYaxasU+ZSozFPg1s2oc3dm2iut0HeABpORCs2bKKx7PHD3u8ebw\nRPpotWrlyclXHv2u5lzbitv15Nt6mAhsdyvEthYH++xxbWr6a7pzbCu2Jfddex3hTR6EiJLHfNfO\n/9YTnRDbkmfdrzuAEDt4WHeAHcW95A8hU3YRHRuBv+sOsaNYl9x37Q3APN05hMi6w3ftRt0hdhTr\nkmfJlF1ExZ91B2iNCSV/ANisO4QoeS/4rh2JTSJ2FPuS+65dC+TlulshOiGSozgYUPKsG4FIvW0h\nSsp6whllJBlRct+1feAR3TlEyfqr79qt3Hc1Gowoedb1ugOIkhQAf9EdYneMKbnv2i8Dr+rOIUrO\nQ75rL9cdYneMKXnWDboDiJKSAa7UHaItppX8QeB93SFEybjbd+33dIdoi1El9107Bfyn7hyiJKSA\nWbpD5MKokgP4rn0fEdpfSxjrNt+1Y3GXXeNKnvUj3QGE0ZqAX+kOkSsjS+679kJkTbsonL/4rv2R\n7hC5MrLkWT8l/I0rRD6tA36pO0R7GFvy7Cq43+vOIYzzH9ldiWLD2JJnXQ18pjuEMMZTvmvP0R2i\nvYwuue/aNcD3dOcQRqgHLtQdoiOMLjmA79oPAffqziFi7wrftT/QHaIjjC951iVAte4QIrZeBf6g\nO0RHlUTJfddeD3xLdw4RSy3ABb5rZ3QH6aiSKDmA79pPAH/UnUPEjuO79mLdITqjZEqe9WMg8hcU\niMh4xHft2O9TUFIl9117C3AWUKc7i4i8VcA3dIfIh5IqOUB26jWd8FpgIVqzBTjDd+1NuoPkQ8mV\nHMB37ccJl70K0Zpv+a79lu4Q+VKSJQfwXfu3wGzdOUTkXOe79j26Q+RTyZY860LgOd0hRGQ8Bji6\nQ+SbCoJAdwatLMfrS7jYYV/dWYRW8wA7ylsrd1Spj+RbF8pMI7yEUJSmhcCpJhYcpOQA+K69FDgO\nWKs7iyi6t4CpvmvX6w5SKCU/Xd+W5XgHAfOBAbqziKJYChzru7bRv9xlJN+G79pLgEnAp5qjiMLz\ngRNMLzhIyXeSvf3sJOATzVFE4SwHjvNd+2PdQYpBSt4K37WXERa9JH4ISszLwNG+a5fMTTik5LuQ\nvb/VsUAkbywvOuRB4PjsOyolQ0q+G75rrwK+DHi6s4hOuwH4d9+1G3UHKTY5u54Dy/ESwDXIevc4\nyhDusFqyO/dKydvBcrzpwC1Ale4sIiebgZm+az+sO4hOUvJ2shxvHPAIsKfuLGK3XgOmx+V+ZYUk\nx+Tt5Lv268A4YIHuLKJVAfBr4BgpeEhG8g6yHE8B/0F4rF6hOY4IfQKc77v2PN1BokRK3knZpbB3\nAmN1ZylxTwDfiNstjIpBSp4HluOVEZ55/zkyqhdbDeGOqv+nO0hUScnzyHK8UYRn38frzlIi7gIu\n811b7ne3G1LyPMseq59DeLNFS28aYy0BfuC79nzdQeJASl4gluN1AS4Gfgb00RzHFGuBK4GbfddO\n6w4TF1LyArMcrxfhvmE/RBbRdNRmwrvfXJu9U61oByl5kViONxT4JTADSGqOExfVwO+Bm0zZA10H\nKXmRWY63F3AR8G2gr+Y4UeUDvwVmZ+96IzpBSq6J5XiVwLnA94ExmuNExbuEq9Xu8107pTuMKaTk\nEWA53rGEZf8qpTeVXwfcD8zxXXuh7jAmkpJHiOV4gwmLfjowESjTm6hgtgCPA3OAp3zXbtGcx2hS\n8ojK3vThFMLCTyb+K+kagReAe4AHfdeu1ZynZEjJY8ByvB6Anf2YAOytN1FOWoBFhFtczwcWluKu\nLFEgJY8hy/GGEZb9KOAIwhN3lVpDQQPhibPnCUv9ou/ach/4CJCSG8ByvHJgNHAw4VJai3C0t4Bh\nQHkeX24tsAJYmf28mLDc7/uuLfd8jyApueGy+9MNISz8YMJVdxWEI3/lNn+uIDzRVwNsyn5s3OHz\nBhmd40dKLoThZPsnIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJy\nIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwn\nJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQwnJRfCcFJyIQz3/zbtcAuj3nkLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train['y'].value_counts().plot.pie(autopct = '%1.2f%%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = train.drop(['cust_group','y','cust_id'],axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test = test.drop(['cust_group','cust_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15000, 157)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 157)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 157)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = pd.concat([x_train,x_test])\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_train = train['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(96,158):\n",
    "    col = 'x'+'_'+str(i)\n",
    "    dummies_df = pd.get_dummies(x[col]).rename(columns=lambda x: col + str(x))\n",
    "    x = pd.concat([x, dummies_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>x_9</th>\n",
       "      <th>x_10</th>\n",
       "      <th>...</th>\n",
       "      <th>x_1561</th>\n",
       "      <th>x_1562</th>\n",
       "      <th>x_1563</th>\n",
       "      <th>x_157-99</th>\n",
       "      <th>x_1571</th>\n",
       "      <th>x_1572</th>\n",
       "      <th>x_1573</th>\n",
       "      <th>x_1574</th>\n",
       "      <th>x_15710</th>\n",
       "      <th>x_15711</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.354167</td>\n",
       "      <td>0.604988</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.012058</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.565979</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.316209</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.008061</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 361 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        x_1       x_2  x_3  x_4  x_5  x_6  x_7  x_8  x_9  x_10   ...     \\\n",
       "0  0.354167  0.604988  -99  -99  -99  -99  -99  -99  -99   -99   ...      \n",
       "1  0.125000  0.012058  -99  -99  -99  -99  -99  -99  -99   -99   ...      \n",
       "2  0.333333  0.565979    0    0    0    0    0    0    0     0   ...      \n",
       "3  0.208333  0.316209    0    0    0    0    1    1    0     0   ...      \n",
       "4  0.208333  0.008061  -99  -99  -99  -99  -99  -99    0     1   ...      \n",
       "\n",
       "   x_1561  x_1562  x_1563  x_157-99  x_1571  x_1572  x_1573  x_1574  x_15710  \\\n",
       "0       0       0       1         1       0       0       0       0        0   \n",
       "1       0       1       0         0       0       1       0       0        0   \n",
       "2       0       1       0         0       0       1       0       0        0   \n",
       "3       0       1       0         0       0       0       0       1        0   \n",
       "4       0       1       0         0       1       0       0       0        0   \n",
       "\n",
       "   x_15711  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  \n",
       "\n",
       "[5 rows x 361 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15000, 361)\n",
      "(10000, 361)\n"
     ]
    }
   ],
   "source": [
    "train_X = x[0:15000]\n",
    "test_X = x[15000:25000]\n",
    "print(train_X.shape)\n",
    "print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cor = train_X\n",
    "# cor['y'] = Y_train\n",
    "# cor.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cor.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# corrr = cor.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# columns = corrr[corrr['y']>0.0].index.values\n",
    "# len(columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train_feature = train_X[columns]\n",
    "# print(train_feature.shape)\n",
    "\n",
    "# train_feature = train_feature.drop(['y'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# column = columns[0:132]\n",
    "# test_feature = test_X[column]\n",
    "# print(test_feature.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import metrics  #accuracy_score,recall_score,f1_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.utils.multiclass import unique_labels\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn import metrics\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.semi_supervised import label_propagation\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, BatchNormalization, Dropout, Reshape, Flatten, MaxPool2D\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D, Conv1D, MaxPooling1D\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train,X_val,y_train,y_val= train_test_split(train_X,Y_train,test_size=0.2,random_state=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def split_data(data_):\n",
    "    idx_1 = data_[data_['label']==0].index\n",
    "    idx_2 = data_[data_['label']==1].index\n",
    "    nb_1 = len(data_.loc[idx_1])\n",
    "    nb_2 = len(data_.loc[idx_2])\n",
    "#     print(nb_1)\n",
    "#     print(nb_2)\n",
    "    idx_list_1 = list(idx_1)\n",
    "    idx_list_2 = list(idx_2)\n",
    "    train_x1 = data_.loc[idx_list_1]\n",
    "    train_x2 = data_.loc[idx_list_2]\n",
    "#     print(train_x1.shape)\n",
    "#     print(train_x2.shape)\n",
    "    return train_x1,train_x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def resample_data(data, number):\n",
    "    idx_1 = data.index\n",
    "    nb_1 = len(idx_1)\n",
    "#     print(nb_1)\n",
    "#     number = int(nb_1 * rate)\n",
    "    idx_1_sub = np.random.choice(idx_1, number)\n",
    "#     print(idx_1_sub)\n",
    "    nb_2 = len(data.loc[idx_1_sub])\n",
    "#     print(nb_2)\n",
    "    idx_list_1 = list(idx_1_sub)\n",
    "    train_1 = data.loc[idx_1_sub]\n",
    "#     print(train_1.shape)\n",
    "    return train_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def concat_data(train_x1, train_x2):\n",
    "    train_data1 = train_x1.drop(['label'],axis =1)\n",
    "    train_y1 = train_x1['label']\n",
    "    \n",
    "    train_data2 = train_x2.drop(['label'],axis =1)\n",
    "    train_y2 = train_x2['label']\n",
    "    \n",
    "    train_data = train_data1.append(train_data2)\n",
    "    train_y = train_y1.append(train_y2)\n",
    "    \n",
    "    return train_data, train_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>x_9</th>\n",
       "      <th>x_10</th>\n",
       "      <th>...</th>\n",
       "      <th>x_1562</th>\n",
       "      <th>x_1563</th>\n",
       "      <th>x_157-99</th>\n",
       "      <th>x_1571</th>\n",
       "      <th>x_1572</th>\n",
       "      <th>x_1573</th>\n",
       "      <th>x_1574</th>\n",
       "      <th>x_15710</th>\n",
       "      <th>x_15711</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.389913</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9804</th>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.425099</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7919</th>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.491195</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4854</th>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.361088</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5947</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.272055</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 362 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           x_1       x_2  x_3  x_4  x_5  x_6  x_7  x_8  x_9  x_10  ...    \\\n",
       "1154  0.291667  0.389913    0    0    0    0    3    2    0     0  ...     \n",
       "9804  0.208333  0.425099    0    0    0    0    4    2    0     0  ...     \n",
       "7919  0.083333  0.491195  -99  -99  -99  -99  -99  -99  -99   -99  ...     \n",
       "4854  0.291667  0.361088  -99  -99  -99  -99  -99  -99  -99   -99  ...     \n",
       "5947  0.250000  0.272055    0    0    0    0    2    2    0     0  ...     \n",
       "\n",
       "      x_1562  x_1563  x_157-99  x_1571  x_1572  x_1573  x_1574  x_15710  \\\n",
       "1154       0       0         0       0       0       0       1        0   \n",
       "9804       1       0         1       0       0       0       0        0   \n",
       "7919       1       0         1       0       0       0       0        0   \n",
       "4854       0       0         1       0       0       0       0        0   \n",
       "5947       1       0         1       0       0       0       0        0   \n",
       "\n",
       "      x_15711  label  \n",
       "1154        0      0  \n",
       "9804        0      1  \n",
       "7919        0      0  \n",
       "4854        0      0  \n",
       "5947        0      0  \n",
       "\n",
       "[5 rows x 362 columns]"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xx = X_train\n",
    "xx['label'] = y_train\n",
    "\n",
    "xx.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# xxx = xx.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# columns = xxx[xxx['label']>0].index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# xxxx = xx[columns]\n",
    "# print(xxxx.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# xxxx.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# column = columns[0:130]\n",
    "# column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# X_val = X_val[column]\n",
    "# print(X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# test_X = test_X[column]\n",
    "# print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11442, 362)\n",
      "(558, 362)\n"
     ]
    }
   ],
   "source": [
    "train_x1, train_x2 = split_data(xx)\n",
    "print(train_x1.shape)\n",
    "print(train_x2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 个模型...\n",
      "0.7086666666666667\n",
      "第 2 个模型...\n",
      "0.7083333333333334\n",
      "第 3 个模型...\n",
      "0.7033333333333334\n",
      "第 4 个模型...\n",
      "0.712\n",
      "第 5 个模型...\n",
      "0.6943333333333334\n",
      "第 6 个模型...\n",
      "0.703\n",
      "第 7 个模型...\n",
      "0.712\n",
      "第 8 个模型...\n",
      "0.7043333333333334\n",
      "第 9 个模型...\n",
      "0.7033333333333334\n",
      "第 10 个模型...\n",
      "0.6963333333333334\n",
      "第 11 个模型...\n",
      "0.7013333333333334\n",
      "第 12 个模型...\n",
      "0.702\n",
      "第 13 个模型...\n",
      "0.7103333333333334\n",
      "第 14 个模型...\n",
      "0.705\n",
      "第 15 个模型...\n",
      "0.7323333333333333\n",
      "第 16 个模型...\n",
      "0.7106666666666667\n",
      "第 17 个模型...\n",
      "0.7076666666666667\n",
      "第 18 个模型...\n",
      "0.708\n",
      "第 19 个模型...\n",
      "0.728\n",
      "第 20 个模型...\n",
      "0.7193333333333334\n",
      "第 21 个模型...\n",
      "0.7036666666666667\n",
      "第 22 个模型...\n",
      "0.7226666666666667\n",
      "第 23 个模型...\n",
      "0.7086666666666667\n",
      "第 24 个模型...\n",
      "0.6953333333333334\n",
      "第 25 个模型...\n",
      "0.7093333333333334\n",
      "第 26 个模型...\n",
      "0.6983333333333334\n",
      "第 27 个模型...\n",
      "0.7036666666666667\n",
      "第 28 个模型...\n",
      "0.6956666666666667\n",
      "第 29 个模型...\n",
      "0.721\n",
      "第 30 个模型...\n",
      "0.7046666666666667\n",
      "第 31 个模型...\n",
      "0.6953333333333334\n",
      "第 32 个模型...\n",
      "0.6876666666666666\n",
      "第 33 个模型...\n",
      "0.719\n",
      "第 34 个模型...\n",
      "0.708\n",
      "第 35 个模型...\n",
      "0.7243333333333334\n",
      "第 36 个模型...\n",
      "0.6986666666666667\n",
      "第 37 个模型...\n",
      "0.717\n",
      "第 38 个模型...\n",
      "0.7123333333333334\n",
      "第 39 个模型...\n",
      "0.7043333333333334\n",
      "第 40 个模型...\n",
      "0.704\n",
      "第 41 个模型...\n",
      "0.6856666666666666\n",
      "第 42 个模型...\n",
      "0.708\n",
      "第 43 个模型...\n",
      "0.7203333333333334\n",
      "第 44 个模型...\n",
      "0.7053333333333334\n",
      "第 45 个模型...\n",
      "0.7156666666666667\n",
      "第 46 个模型...\n",
      "0.7166666666666667\n",
      "第 47 个模型...\n",
      "0.7106666666666667\n",
      "第 48 个模型...\n",
      "0.718\n",
      "第 49 个模型...\n",
      "0.708\n",
      "第 50 个模型...\n",
      "0.7116666666666667\n",
      "第 51 个模型...\n",
      "0.7063333333333334\n",
      "第 52 个模型...\n",
      "0.7003333333333334\n",
      "第 53 个模型...\n",
      "0.7033333333333334\n",
      "第 54 个模型...\n",
      "0.7123333333333334\n",
      "第 55 个模型...\n",
      "0.701\n",
      "第 56 个模型...\n",
      "0.7133333333333334\n",
      "第 57 个模型...\n",
      "0.7076666666666667\n",
      "第 58 个模型...\n",
      "0.7143333333333334\n",
      "第 59 个模型...\n",
      "0.6996666666666667\n",
      "第 60 个模型...\n",
      "0.708\n",
      "第 61 个模型...\n",
      "0.7186666666666667\n",
      "第 62 个模型...\n",
      "0.716\n",
      "第 63 个模型...\n",
      "0.6966666666666667\n",
      "第 64 个模型...\n",
      "0.6933333333333334\n",
      "第 65 个模型...\n",
      "0.7086666666666667\n",
      "第 66 个模型...\n",
      "0.716\n",
      "第 67 个模型...\n",
      "0.715\n",
      "第 68 个模型...\n",
      "0.71\n",
      "第 69 个模型...\n",
      "0.7176666666666667\n",
      "第 70 个模型...\n",
      "0.7146666666666667\n",
      "第 71 个模型...\n",
      "0.7216666666666667\n",
      "第 72 个模型...\n",
      "0.711\n",
      "第 73 个模型...\n",
      "0.698\n",
      "第 74 个模型...\n",
      "0.687\n",
      "第 75 个模型...\n",
      "0.712\n",
      "第 76 个模型...\n",
      "0.712\n",
      "第 77 个模型...\n",
      "0.711\n",
      "第 78 个模型...\n",
      "0.689\n",
      "第 79 个模型...\n",
      "0.7086666666666667\n",
      "第 80 个模型...\n",
      "0.7016666666666667\n",
      "第 81 个模型...\n",
      "0.7136666666666667\n",
      "第 82 个模型...\n",
      "0.7023333333333334\n",
      "第 83 个模型...\n",
      "0.7166666666666667\n",
      "第 84 个模型...\n",
      "0.7026666666666667\n",
      "第 85 个模型...\n",
      "0.6973333333333334\n",
      "第 86 个模型...\n",
      "0.6886666666666666\n",
      "第 87 个模型...\n",
      "0.7016666666666667\n",
      "第 88 个模型...\n",
      "0.7153333333333334\n",
      "第 89 个模型...\n",
      "0.708\n",
      "第 90 个模型...\n",
      "0.7036666666666667\n",
      "第 91 个模型...\n",
      "0.7093333333333334\n",
      "第 92 个模型...\n",
      "0.7096666666666667\n",
      "第 93 个模型...\n",
      "0.704\n",
      "第 94 个模型...\n",
      "0.681\n",
      "第 95 个模型...\n",
      "0.6853333333333333\n",
      "第 96 个模型...\n",
      "0.6993333333333334\n",
      "第 97 个模型...\n",
      "0.7176666666666667\n",
      "第 98 个模型...\n",
      "0.707\n",
      "第 99 个模型...\n",
      "0.716\n",
      "第 100 个模型...\n",
      "0.7223333333333334\n",
      "第 101 个模型...\n",
      "0.7236666666666667\n",
      "第 102 个模型...\n",
      "0.7103333333333334\n",
      "第 103 个模型...\n",
      "0.7016666666666667\n",
      "第 104 个模型...\n",
      "0.7153333333333334\n",
      "第 105 个模型...\n",
      "0.709\n",
      "第 106 个模型...\n",
      "0.7076666666666667\n",
      "第 107 个模型...\n",
      "0.707\n",
      "第 108 个模型...\n",
      "0.7076666666666667\n",
      "第 109 个模型...\n",
      "0.699\n",
      "第 110 个模型...\n",
      "0.6836666666666666\n",
      "第 111 个模型...\n",
      "0.708\n",
      "第 112 个模型...\n",
      "0.708\n",
      "第 113 个模型...\n",
      "0.7226666666666667\n",
      "第 114 个模型...\n",
      "0.7033333333333334\n",
      "第 115 个模型...\n",
      "0.6926666666666667\n",
      "第 116 个模型...\n",
      "0.7046666666666667\n",
      "第 117 个模型...\n",
      "0.703\n",
      "第 118 个模型...\n",
      "0.706\n",
      "第 119 个模型...\n",
      "0.7123333333333334\n",
      "第 120 个模型...\n",
      "0.6886666666666666\n",
      "第 121 个模型...\n",
      "0.7136666666666667\n",
      "第 122 个模型...\n",
      "0.6853333333333333\n",
      "第 123 个模型...\n",
      "0.7126666666666667\n",
      "第 124 个模型...\n",
      "0.708\n",
      "第 125 个模型...\n",
      "0.7053333333333334\n",
      "第 126 个模型...\n",
      "0.7063333333333334\n",
      "第 127 个模型...\n",
      "0.7003333333333334\n",
      "第 128 个模型...\n",
      "0.705\n",
      "第 129 个模型...\n",
      "0.71\n",
      "第 130 个模型...\n",
      "0.7083333333333334\n",
      "第 131 个模型...\n",
      "0.6883333333333334\n",
      "第 132 个模型...\n",
      "0.717\n",
      "第 133 个模型...\n",
      "0.7293333333333333\n",
      "第 134 个模型...\n",
      "0.7236666666666667\n",
      "第 135 个模型...\n",
      "0.7153333333333334\n",
      "第 136 个模型...\n",
      "0.7143333333333334\n",
      "第 137 个模型...\n",
      "0.7043333333333334\n",
      "第 138 个模型...\n",
      "0.7123333333333334\n",
      "第 139 个模型...\n",
      "0.7223333333333334\n",
      "第 140 个模型...\n",
      "0.7073333333333334\n",
      "第 141 个模型...\n",
      "0.6983333333333334\n",
      "第 142 个模型...\n",
      "0.687\n",
      "第 143 个模型...\n",
      "0.71\n",
      "第 144 个模型...\n",
      "0.7106666666666667\n",
      "第 145 个模型...\n",
      "0.6986666666666667\n",
      "第 146 个模型...\n",
      "0.694\n",
      "第 147 个模型...\n",
      "0.707\n",
      "第 148 个模型...\n",
      "0.7146666666666667\n",
      "第 149 个模型...\n",
      "0.7216666666666667\n",
      "第 150 个模型...\n",
      "0.7063333333333334\n",
      "第 151 个模型...\n",
      "0.7073333333333334\n",
      "第 152 个模型...\n",
      "0.697\n",
      "第 153 个模型...\n",
      "0.6946666666666667\n",
      "第 154 个模型...\n",
      "0.698\n",
      "第 155 个模型...\n",
      "0.7176666666666667\n",
      "第 156 个模型...\n",
      "0.7\n",
      "第 157 个模型...\n",
      "0.7076666666666667\n",
      "第 158 个模型...\n",
      "0.7106666666666667\n",
      "第 159 个模型...\n",
      "0.7166666666666667\n",
      "第 160 个模型...\n",
      "0.6923333333333334\n",
      "第 161 个模型...\n",
      "0.7126666666666667\n",
      "第 162 个模型...\n",
      "0.71\n",
      "第 163 个模型...\n",
      "0.7143333333333334\n",
      "第 164 个模型...\n",
      "0.7056666666666667\n",
      "第 165 个模型...\n",
      "0.6903333333333334\n",
      "第 166 个模型...\n",
      "0.7226666666666667\n",
      "第 167 个模型...\n",
      "0.7043333333333334\n",
      "第 168 个模型...\n",
      "0.7026666666666667\n",
      "第 169 个模型...\n",
      "0.71\n",
      "第 170 个模型...\n",
      "0.6896666666666667\n",
      "第 171 个模型...\n",
      "0.7086666666666667\n",
      "第 172 个模型...\n",
      "0.7006666666666667\n",
      "第 173 个模型...\n",
      "0.7223333333333334\n",
      "第 174 个模型...\n",
      "0.7136666666666667\n",
      "第 175 个模型...\n",
      "0.7136666666666667\n",
      "第 176 个模型...\n",
      "0.7113333333333334\n",
      "第 177 个模型...\n",
      "0.6903333333333334\n",
      "第 178 个模型...\n",
      "0.6896666666666667\n",
      "第 179 个模型...\n",
      "0.6923333333333334\n",
      "第 180 个模型...\n",
      "0.6913333333333334\n",
      "第 181 个模型...\n",
      "0.7126666666666667\n",
      "第 182 个模型...\n",
      "0.719\n",
      "第 183 个模型...\n",
      "0.706\n",
      "第 184 个模型...\n",
      "0.7173333333333334\n",
      "第 185 个模型...\n",
      "0.7006666666666667\n",
      "第 186 个模型...\n",
      "0.703\n",
      "第 187 个模型...\n",
      "0.7156666666666667\n",
      "第 188 个模型...\n",
      "0.7066666666666667\n",
      "第 189 个模型...\n",
      "0.7\n",
      "第 190 个模型...\n",
      "0.706\n",
      "第 191 个模型...\n",
      "0.7053333333333334\n",
      "第 192 个模型...\n",
      "0.7013333333333334\n",
      "第 193 个模型...\n",
      "0.7043333333333334\n",
      "第 194 个模型...\n",
      "0.7013333333333334\n",
      "第 195 个模型...\n",
      "0.7103333333333334\n",
      "第 196 个模型...\n",
      "0.6983333333333334\n",
      "第 197 个模型...\n",
      "0.7073333333333334\n",
      "第 198 个模型...\n",
      "0.7036666666666667\n",
      "第 199 个模型...\n",
      "0.7\n",
      "第 200 个模型...\n",
      "0.6963333333333334\n"
     ]
    }
   ],
   "source": [
    "pred = []\n",
    "test_pred = [] \n",
    "for i in range(200):\n",
    "    print('第',i+1,'个模型...')\n",
    "    train_temp = resample_data(train_x1, 600)\n",
    "#     print(train_temp.shape)\n",
    "    multi_x, multi_y= concat_data(train_temp, train_x2)\n",
    "#     print(multi_x.shape)\n",
    "#     print(multi_y.shape)\n",
    "    \n",
    "    gbm = XGBClassifier( n_estimators= 150, max_depth= 5, min_child_weight= 2, gamma=0.9, subsample=0.8, \n",
    "                        colsample_bytree=0.8, objective= 'binary:logistic', nthread= -1, scale_pos_weight=1).fit(multi_x, multi_y)\n",
    "    \n",
    "    predictions = gbm.predict(X_val)\n",
    "    \n",
    "    test_predictions = gbm.predict(test_X)\n",
    "\n",
    "    target_names = ['class 0', 'class 1']\n",
    "#     print(classification_report(y_val, predictions, target_names=target_names))\n",
    "\n",
    "    val_acc = metrics.accuracy_score(y_val,predictions)#验证集上的auc值\n",
    "    print(val_acc)\n",
    "    \n",
    "    pred.append(predictions)\n",
    "    test_pred.append(test_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[189   7  95 ...   0 192   0]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "    class 0       1.00      0.36      0.53      2867\n",
      "    class 1       0.06      0.96      0.12       133\n",
      "\n",
      "avg / total       0.95      0.38      0.51      3000\n",
      "\n",
      "0.384\n",
      "1971\n"
     ]
    }
   ],
   "source": [
    "pred_sum = pred[0]\n",
    "for i in range(1,200):\n",
    "    pred_sum = pred_sum + pred[i]\n",
    "print(pred_sum)\n",
    "for i in range(3000):\n",
    "    if pred_sum[i] >= 1:\n",
    "        pred_sum[i] = 1\n",
    "    else:\n",
    "        pred_sum[i] = 0\n",
    "\n",
    "target_names = ['class 0', 'class 1']\n",
    "print(classification_report(y_val, pred_sum, target_names=target_names))\n",
    "\n",
    "val_acc = metrics.accuracy_score(y_val,pred_sum)#验证集上的auc值\n",
    "print(val_acc)\n",
    "print(np.sum(pred_sum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[136   0 199 ...  88 187 191]\n",
      "8674\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "test_pred_sum = test_pred[0]\n",
    "for i in range(1,200):\n",
    "    test_pred_sum = test_pred_sum + test_pred[i]\n",
    "print(test_pred_sum)\n",
    "for i in range(10000):\n",
    "    if test_pred_sum[i] >= 1:\n",
    "        test_pred_sum[i] = 1\n",
    "    else:\n",
    "        test_pred_sum[i] = 0\n",
    "\n",
    "print(np.sum(test_pred_sum))\n",
    "print(test_pred_sum.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['label'] = test_pred_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cust_id</th>\n",
       "      <th>cust_group</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>...</th>\n",
       "      <th>x_149</th>\n",
       "      <th>x_150</th>\n",
       "      <th>x_151</th>\n",
       "      <th>x_152</th>\n",
       "      <th>x_153</th>\n",
       "      <th>x_154</th>\n",
       "      <th>x_155</th>\n",
       "      <th>x_156</th>\n",
       "      <th>x_157</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.659675</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100001</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.657454</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-99</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100002</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>0.825764</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-99</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100003</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>0.473441</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100004</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.344635</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 160 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cust_id cust_group       x_1       x_2  x_3  x_4  x_5  x_6  x_7  x_8  \\\n",
       "0   100000    group_3  0.125000  0.659675    0    1    1    0    3    3   \n",
       "1   100001    group_3  0.250000  0.657454    0    0    0    0    0    0   \n",
       "2   100002    group_3  0.604167  0.825764    0    0    0    0    3    3   \n",
       "3   100003    group_3  0.312500  0.473441    0    0    0    0    0    0   \n",
       "4   100004    group_3  0.458333  0.344635  -99  -99  -99  -99  -99  -99   \n",
       "\n",
       "   ...    x_149  x_150  x_151  x_152  x_153  x_154  x_155  x_156  x_157  label  \n",
       "0  ...        1      1      1      1      1      1      1      2      2      1  \n",
       "1  ...        1      4      1      1      1      1      1      2    -99      1  \n",
       "2  ...        1      1      1      1      1      1      1      2    -99      1  \n",
       "3  ...        1      1      1      1      1      2      2      2      2      1  \n",
       "4  ...        1      2      1      1      1      1      1      2      1      1  \n",
       "\n",
       "[5 rows x 160 columns]"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1551, 160)\n",
      "(8449, 160)\n"
     ]
    }
   ],
   "source": [
    "test_x1,test_x2 = split_data(test)\n",
    "print(test_x1.shape)\n",
    "print(test_x2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_x1.to_csv('../data/train_x_y1.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pretreatment(train, test):\n",
    "    x_test = test.drop(['cust_group','cust_id'],axis=1)\n",
    "    x_train = train.drop(['cust_group','y','cust_id'],axis =1)\n",
    "    \n",
    "    x = pd.concat([x_train,x_test])\n",
    "    print(x.shape)\n",
    "    \n",
    "    for i in range(96,158):\n",
    "        col = 'x'+'_'+str(i)\n",
    "        dummies_df = pd.get_dummies(x[col]).rename(columns=lambda x: col + str(x))\n",
    "        x = pd.concat([x, dummies_df], axis=1)\n",
    "    \n",
    "    train_X = x[0:15000]\n",
    "    test_X = x[15000:25000]\n",
    "    print(train_X.shape)\n",
    "    print(test_X.shape)\n",
    "    \n",
    "    X_train,X_val,y_train,y_val= train_test_split(train_X,Y_train,test_size=0.2,random_state=2)\n",
    "    \n",
    "    xx = X_train\n",
    "    xx['label'] = y_train\n",
    "\n",
    "    xxx = xx.corr()\n",
    "    columns = xxx[xxx['label']>0].index.values\n",
    "#     print(columns)\n",
    "    \n",
    "    xxxx = xx[columns]\n",
    "    number = xxxx.shape[1]\n",
    "    \n",
    "    column = columns[1:number]\n",
    "    \n",
    "    X_val = X_val[column]\n",
    "    print(X_val.shape)\n",
    "\n",
    "    test_X = test_X[column]\n",
    "    print(test_X.shape)\n",
    "\n",
    "    return xxxx,X_val,test_X,y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23449, 158)\n",
      "(15000, 359)\n",
      "(8449, 359)\n",
      "(3000, 130)\n",
      "(8449, 130)\n",
      "(11442, 131)\n",
      "(558, 131)\n"
     ]
    }
   ],
   "source": [
    "xxxx_2,X_val_2,test_X_2,y_val_2 = pretreatment(train, test_x2)\n",
    "train_x1_2, train_x2_2 = split_data(xxxx_2)\n",
    "print(train_x1_2.shape)\n",
    "print(train_x2_2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_10</th>\n",
       "      <th>x_100</th>\n",
       "      <th>x_101</th>\n",
       "      <th>x_104</th>\n",
       "      <th>x_105</th>\n",
       "      <th>x_11</th>\n",
       "      <th>x_12</th>\n",
       "      <th>x_121</th>\n",
       "      <th>x_122</th>\n",
       "      <th>x_124</th>\n",
       "      <th>...</th>\n",
       "      <th>x_153-99</th>\n",
       "      <th>x_1532</th>\n",
       "      <th>x_1533</th>\n",
       "      <th>x_1542</th>\n",
       "      <th>x_1543</th>\n",
       "      <th>x_1552</th>\n",
       "      <th>x_1553</th>\n",
       "      <th>x_156-99</th>\n",
       "      <th>x_1562</th>\n",
       "      <th>x_157-99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7592</th>\n",
       "      <td>-99</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3551</th>\n",
       "      <td>-99</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9698</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3759</th>\n",
       "      <td>-99</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2353</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      x_10  x_100  x_101  x_104  x_105  x_11  x_12  x_121  x_122  x_124  \\\n",
       "7592   -99      1      1    -99    -99   -99   -99    -99    -99    -99   \n",
       "3551   -99      1      1    -99    -99   -99   -99    -99    -99    -99   \n",
       "9698     0      1      1    -99    -99     0     0    -99    -99    -99   \n",
       "3759   -99      1      1    -99    -99   -99   -99    -99    -99    -99   \n",
       "2353     1      1      1    -99    -99     1     0    -99    -99    -99   \n",
       "\n",
       "        ...     x_153-99  x_1532  x_1533  x_1542  x_1543  x_1552  x_1553  \\\n",
       "7592    ...            0       0       1       0       1       0       1   \n",
       "3551    ...            0       0       0       0       0       0       0   \n",
       "9698    ...            0       0       0       1       0       1       0   \n",
       "3759    ...            0       0       0       0       0       0       0   \n",
       "2353    ...            0       0       0       0       0       0       0   \n",
       "\n",
       "      x_156-99  x_1562  x_157-99  \n",
       "7592         0       1         1  \n",
       "3551         0       0         0  \n",
       "9698         0       1         0  \n",
       "3759         0       1         0  \n",
       "2353         0       1         0  \n",
       "\n",
       "[5 rows x 130 columns]"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 个模型...\n",
      "0.698\n",
      "第 2 个模型...\n",
      "0.7006666666666667\n",
      "第 3 个模型...\n",
      "0.693\n",
      "第 4 个模型...\n",
      "0.708\n",
      "第 5 个模型...\n",
      "0.686\n",
      "第 6 个模型...\n",
      "0.713\n",
      "第 7 个模型...\n",
      "0.7106666666666667\n",
      "第 8 个模型...\n",
      "0.7163333333333334\n",
      "第 9 个模型...\n",
      "0.7056666666666667\n",
      "第 10 个模型...\n",
      "0.694\n",
      "第 11 个模型...\n",
      "0.6896666666666667\n",
      "第 12 个模型...\n",
      "0.692\n",
      "第 13 个模型...\n",
      "0.719\n",
      "第 14 个模型...\n",
      "0.721\n",
      "第 15 个模型...\n",
      "0.679\n",
      "第 16 个模型...\n",
      "0.7063333333333334\n",
      "第 17 个模型...\n",
      "0.7126666666666667\n",
      "第 18 个模型...\n",
      "0.7223333333333334\n",
      "第 19 个模型...\n",
      "0.7006666666666667\n",
      "第 20 个模型...\n",
      "0.7116666666666667\n",
      "第 21 个模型...\n",
      "0.7143333333333334\n",
      "第 22 个模型...\n",
      "0.712\n",
      "第 23 个模型...\n",
      "0.671\n",
      "第 24 个模型...\n",
      "0.699\n",
      "第 25 个模型...\n",
      "0.6876666666666666\n",
      "第 26 个模型...\n",
      "0.7073333333333334\n",
      "第 27 个模型...\n",
      "0.6893333333333334\n",
      "第 28 个模型...\n",
      "0.6853333333333333\n",
      "第 29 个模型...\n",
      "0.6856666666666666\n",
      "第 30 个模型...\n",
      "0.72\n",
      "第 31 个模型...\n",
      "0.7003333333333334\n",
      "第 32 个模型...\n",
      "0.7113333333333334\n",
      "第 33 个模型...\n",
      "0.7096666666666667\n",
      "第 34 个模型...\n",
      "0.7163333333333334\n",
      "第 35 个模型...\n",
      "0.682\n",
      "第 36 个模型...\n",
      "0.7283333333333334\n",
      "第 37 个模型...\n",
      "0.7036666666666667\n",
      "第 38 个模型...\n",
      "0.7013333333333334\n",
      "第 39 个模型...\n",
      "0.6963333333333334\n",
      "第 40 个模型...\n",
      "0.7146666666666667\n",
      "第 41 个模型...\n",
      "0.6923333333333334\n",
      "第 42 个模型...\n",
      "0.7026666666666667\n",
      "第 43 个模型...\n",
      "0.701\n",
      "第 44 个模型...\n",
      "0.7033333333333334\n",
      "第 45 个模型...\n",
      "0.718\n",
      "第 46 个模型...\n",
      "0.7076666666666667\n",
      "第 47 个模型...\n",
      "0.7086666666666667\n",
      "第 48 个模型...\n",
      "0.716\n",
      "第 49 个模型...\n",
      "0.7\n",
      "第 50 个模型...\n",
      "0.708\n",
      "第 51 个模型...\n",
      "0.7066666666666667\n",
      "第 52 个模型...\n",
      "0.7306666666666667\n",
      "第 53 个模型...\n",
      "0.7133333333333334\n",
      "第 54 个模型...\n",
      "0.718\n",
      "第 55 个模型...\n",
      "0.7106666666666667\n",
      "第 56 个模型...\n",
      "0.7296666666666667\n",
      "第 57 个模型...\n",
      "0.7126666666666667\n",
      "第 58 个模型...\n",
      "0.707\n",
      "第 59 个模型...\n",
      "0.701\n",
      "第 60 个模型...\n",
      "0.7156666666666667\n",
      "第 61 个模型...\n",
      "0.6826666666666666\n",
      "第 62 个模型...\n",
      "0.716\n",
      "第 63 个模型...\n",
      "0.7116666666666667\n",
      "第 64 个模型...\n",
      "0.71\n",
      "第 65 个模型...\n",
      "0.7063333333333334\n",
      "第 66 个模型...\n",
      "0.7086666666666667\n",
      "第 67 个模型...\n",
      "0.6973333333333334\n",
      "第 68 个模型...\n",
      "0.7156666666666667\n",
      "第 69 个模型...\n",
      "0.691\n",
      "第 70 个模型...\n",
      "0.7063333333333334\n",
      "第 71 个模型...\n",
      "0.7063333333333334\n",
      "第 72 个模型...\n",
      "0.709\n",
      "第 73 个模型...\n",
      "0.6926666666666667\n",
      "第 74 个模型...\n",
      "0.7256666666666667\n",
      "第 75 个模型...\n",
      "0.7273333333333334\n",
      "第 76 个模型...\n",
      "0.7216666666666667\n",
      "第 77 个模型...\n",
      "0.6876666666666666\n",
      "第 78 个模型...\n",
      "0.728\n",
      "第 79 个模型...\n",
      "0.708\n",
      "第 80 个模型...\n",
      "0.7293333333333333\n",
      "第 81 个模型...\n",
      "0.7076666666666667\n",
      "第 82 个模型...\n",
      "0.7053333333333334\n",
      "第 83 个模型...\n",
      "0.7006666666666667\n",
      "第 84 个模型...\n",
      "0.7043333333333334\n",
      "第 85 个模型...\n",
      "0.6993333333333334\n",
      "第 86 个模型...\n",
      "0.711\n",
      "第 87 个模型...\n",
      "0.7033333333333334\n",
      "第 88 个模型...\n",
      "0.6963333333333334\n",
      "第 89 个模型...\n",
      "0.6913333333333334\n",
      "第 90 个模型...\n",
      "0.6783333333333333\n",
      "第 91 个模型...\n",
      "0.7136666666666667\n",
      "第 92 个模型...\n",
      "0.693\n",
      "第 93 个模型...\n",
      "0.718\n",
      "第 94 个模型...\n",
      "0.7036666666666667\n",
      "第 95 个模型...\n",
      "0.7086666666666667\n",
      "第 96 个模型...\n",
      "0.7086666666666667\n",
      "第 97 个模型...\n",
      "0.7103333333333334\n",
      "第 98 个模型...\n",
      "0.7156666666666667\n",
      "第 99 个模型...\n",
      "0.7003333333333334\n",
      "第 100 个模型...\n",
      "0.7033333333333334\n",
      "第 101 个模型...\n",
      "0.7143333333333334\n",
      "第 102 个模型...\n",
      "0.6976666666666667\n",
      "第 103 个模型...\n",
      "0.7003333333333334\n",
      "第 104 个模型...\n",
      "0.7283333333333334\n",
      "第 105 个模型...\n",
      "0.695\n",
      "第 106 个模型...\n",
      "0.7143333333333334\n",
      "第 107 个模型...\n",
      "0.7086666666666667\n",
      "第 108 个模型...\n",
      "0.7156666666666667\n",
      "第 109 个模型...\n",
      "0.6933333333333334\n",
      "第 110 个模型...\n",
      "0.696\n",
      "第 111 个模型...\n",
      "0.7116666666666667\n",
      "第 112 个模型...\n",
      "0.7096666666666667\n",
      "第 113 个模型...\n",
      "0.713\n",
      "第 114 个模型...\n",
      "0.7043333333333334\n",
      "第 115 个模型...\n",
      "0.706\n",
      "第 116 个模型...\n",
      "0.72\n",
      "第 117 个模型...\n",
      "0.699\n",
      "第 118 个模型...\n",
      "0.6873333333333334\n",
      "第 119 个模型...\n",
      "0.703\n",
      "第 120 个模型...\n",
      "0.718\n",
      "第 121 个模型...\n",
      "0.702\n",
      "第 122 个模型...\n",
      "0.6903333333333334\n",
      "第 123 个模型...\n",
      "0.7113333333333334\n",
      "第 124 个模型...\n",
      "0.704\n",
      "第 125 个模型...\n",
      "0.7216666666666667\n",
      "第 126 个模型...\n",
      "0.7093333333333334\n",
      "第 127 个模型...\n",
      "0.7013333333333334\n",
      "第 128 个模型...\n",
      "0.7206666666666667\n",
      "第 129 个模型...\n",
      "0.736\n",
      "第 130 个模型...\n",
      "0.709\n",
      "第 131 个模型...\n",
      "0.694\n",
      "第 132 个模型...\n",
      "0.688\n",
      "第 133 个模型...\n",
      "0.7123333333333334\n",
      "第 134 个模型...\n",
      "0.731\n",
      "第 135 个模型...\n",
      "0.7086666666666667\n",
      "第 136 个模型...\n",
      "0.7046666666666667\n",
      "第 137 个模型...\n",
      "0.706\n",
      "第 138 个模型...\n",
      "0.6926666666666667\n",
      "第 139 个模型...\n",
      "0.695\n",
      "第 140 个模型...\n",
      "0.6986666666666667\n",
      "第 141 个模型...\n",
      "0.6723333333333333\n",
      "第 142 个模型...\n",
      "0.7003333333333334\n",
      "第 143 个模型...\n",
      "0.6933333333333334\n",
      "第 144 个模型...\n",
      "0.7126666666666667\n",
      "第 145 个模型...\n",
      "0.6996666666666667\n",
      "第 146 个模型...\n",
      "0.711\n",
      "第 147 个模型...\n",
      "0.6806666666666666\n",
      "第 148 个模型...\n",
      "0.702\n",
      "第 149 个模型...\n",
      "0.721\n",
      "第 150 个模型...\n",
      "0.6996666666666667\n",
      "第 151 个模型...\n",
      "0.693\n",
      "第 152 个模型...\n",
      "0.697\n",
      "第 153 个模型...\n",
      "0.7103333333333334\n",
      "第 154 个模型...\n",
      "0.7036666666666667\n",
      "第 155 个模型...\n",
      "0.721\n",
      "第 156 个模型...\n",
      "0.7126666666666667\n",
      "第 157 个模型...\n",
      "0.6946666666666667\n",
      "第 158 个模型...\n",
      "0.7013333333333334\n",
      "第 159 个模型...\n",
      "0.7016666666666667\n",
      "第 160 个模型...\n",
      "0.7053333333333334\n",
      "第 161 个模型...\n",
      "0.6873333333333334\n",
      "第 162 个模型...\n",
      "0.7266666666666667\n",
      "第 163 个模型...\n",
      "0.71\n",
      "第 164 个模型...\n",
      "0.697\n",
      "第 165 个模型...\n",
      "0.7253333333333334\n",
      "第 166 个模型...\n",
      "0.7013333333333334\n",
      "第 167 个模型...\n",
      "0.6886666666666666\n",
      "第 168 个模型...\n",
      "0.705\n",
      "第 169 个模型...\n",
      "0.707\n",
      "第 170 个模型...\n",
      "0.7056666666666667\n",
      "第 171 个模型...\n",
      "0.7176666666666667\n",
      "第 172 个模型...\n",
      "0.7123333333333334\n",
      "第 173 个模型...\n",
      "0.704\n",
      "第 174 个模型...\n",
      "0.7073333333333334\n",
      "第 175 个模型...\n",
      "0.7073333333333334\n",
      "第 176 个模型...\n",
      "0.702\n",
      "第 177 个模型...\n",
      "0.685\n",
      "第 178 个模型...\n",
      "0.6836666666666666\n",
      "第 179 个模型...\n",
      "0.727\n",
      "第 180 个模型...\n",
      "0.6946666666666667\n",
      "第 181 个模型...\n",
      "0.689\n",
      "第 182 个模型...\n",
      "0.7103333333333334\n",
      "第 183 个模型...\n",
      "0.687\n",
      "第 184 个模型...\n",
      "0.709\n",
      "第 185 个模型...\n",
      "0.7063333333333334\n",
      "第 186 个模型...\n",
      "0.6986666666666667\n",
      "第 187 个模型...\n",
      "0.7173333333333334\n",
      "第 188 个模型...\n",
      "0.6993333333333334\n",
      "第 189 个模型...\n",
      "0.691\n",
      "第 190 个模型...\n",
      "0.695\n",
      "第 191 个模型...\n",
      "0.703\n",
      "第 192 个模型...\n",
      "0.7073333333333334\n",
      "第 193 个模型...\n",
      "0.6926666666666667\n",
      "第 194 个模型...\n",
      "0.7273333333333334\n",
      "第 195 个模型...\n",
      "0.7316666666666667\n",
      "第 196 个模型...\n",
      "0.7076666666666667\n",
      "第 197 个模型...\n",
      "0.7096666666666667\n",
      "第 198 个模型...\n",
      "0.7013333333333334\n",
      "第 199 个模型...\n",
      "0.7003333333333334\n",
      "第 200 个模型...\n",
      "0.694\n"
     ]
    }
   ],
   "source": [
    "pred_2 = []\n",
    "test_pred_2 = [] \n",
    "for i in range(200):\n",
    "    print('第',i+1,'个模型...')\n",
    "    train_temp = resample_data(train_x1_2, 600)\n",
    "#     print(train_temp.shape)\n",
    "    multi_x, multi_y= concat_data(train_temp, train_x2_2)\n",
    "#     print(multi_x.shape)\n",
    "#     print(multi_y.shape)\n",
    "    \n",
    "    gbm = GradientBoostingClassifier(n_estimators=100, learning_rate=0.05,max_depth=4, random_state=0).fit(multi_x, multi_y)\n",
    "    \n",
    "    predictions = gbm.predict(X_val_2)\n",
    "    \n",
    "    test_predictions = gbm.predict(test_X_2)\n",
    "\n",
    "    target_names = ['class 0', 'class 1']\n",
    "#     print(classification_report(y_val_2, predictions, target_names=target_names))\n",
    "\n",
    "    val_acc = metrics.accuracy_score(y_val_2,predictions)#验证集上的auc值\n",
    "    print(val_acc)\n",
    "    \n",
    "    pred_2.append(predictions)\n",
    "    test_pred_2.append(test_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[197   0   4 ...   0 189   0]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "    class 0       0.96      0.93      0.95      2867\n",
      "    class 1       0.15      0.25      0.18       133\n",
      "\n",
      "avg / total       0.93      0.90      0.91      3000\n",
      "\n",
      "0.902\n",
      "227\n"
     ]
    }
   ],
   "source": [
    "pred_sum2 = pred_2[0]\n",
    "for i in range(1,200):\n",
    "    pred_sum2 = pred_sum2 + pred_2[i]\n",
    "print(pred_sum2)\n",
    "for i in range(3000):\n",
    "    if pred_sum2[i] >= 200:\n",
    "        pred_sum2[i] = 1\n",
    "    else:\n",
    "        pred_sum2[i] = 0\n",
    "\n",
    "target_names = ['class 0', 'class 1']\n",
    "print(classification_report(y_val_2, pred_sum2, target_names=target_names))\n",
    "\n",
    "val_acc = metrics.accuracy_score(y_val_2, pred_sum2)#验证集上的auc值\n",
    "print(val_acc)\n",
    "print(np.sum(pred_sum2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[160  96 200 ...  45 188 197]\n",
      "1283\n",
      "(8449,)\n"
     ]
    }
   ],
   "source": [
    "test_pred_sum2 = test_pred_2[0]\n",
    "for i in range(1,200):\n",
    "    test_pred_sum2 = test_pred_sum2 + test_pred_2[i]\n",
    "print(test_pred_sum2)\n",
    "for i in range(8449):\n",
    "    if test_pred_sum2[i] >= 200:\n",
    "        test_pred_sum2[i] = 1\n",
    "    else:\n",
    "        test_pred_sum2[i] = 0\n",
    "\n",
    "print(np.sum(test_pred_sum2))\n",
    "print(test_pred_sum2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(473, 160)\n",
      "(7976, 160)\n"
     ]
    }
   ],
   "source": [
    "test_x2['label'] = test_pred_sum2\n",
    "\n",
    "test_x3,test_x4 = split_data(test_x2)\n",
    "print(test_x3.shape)\n",
    "print(test_x4.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_x3.to_csv('../data/train_x_y2.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pred_sum3 = pred3[0]\n",
    "# for i in range(1,200):\n",
    "#     pred_sum3 = pred_sum3 + pred3[i]\n",
    "# print(pred_sum)\n",
    "# for i in range(3000):\n",
    "#     if pred_sum3[i] >= 1:\n",
    "#         pred_sum3[i] = 1\n",
    "#     else:\n",
    "#         pred_sum3[i] = 0\n",
    "\n",
    "# target_names = ['class 0', 'class 1']\n",
    "# print(classification_report(y_val, pred_sum3, target_names=target_names))\n",
    "\n",
    "# val_acc = metrics.accuracy_score(y_val,pred_sum3)#验证集上的auc值\n",
    "# print(val_acc)\n",
    "# print(np.sum(pred_sum3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pred4 = []\n",
    "# for i in range(1):\n",
    "#     print('第',i+1,'个模型...')\n",
    "#     train_temp = resample_data(train_x1, 600)\n",
    "# #     print(train_temp.shape)\n",
    "#     multi_x, multi_y= concat_data(train_temp, train_x2)\n",
    "# #     print(multi_x.shape)\n",
    "# #     print(multi_y.shape)\n",
    "    \n",
    "#     clf = SVC(kernel='linear', C=1).fit(multi_x, multi_y)\n",
    "    \n",
    "#     predictions = clf.predict(X_val)\n",
    "\n",
    "#     target_names = ['class 0', 'class 1']\n",
    "#     print(classification_report(y_val, predictions, target_names=target_names))\n",
    "\n",
    "#     val_acc = metrics.accuracy_score(y_val,predictions)#验证集上的auc值\n",
    "#     print(val_acc)\n",
    "    \n",
    "#     pred4.append(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pred_sum4 = pred4[0]\n",
    "# for i in range(1,200):\n",
    "#     pred_sum4 = pred_sum4 + pred4[i]\n",
    "# print(pred_sum4)\n",
    "# for i in range(3000):\n",
    "#     if pred_sum4[i] >= 180:\n",
    "#         pred_sum4[i] = 1\n",
    "#     else:\n",
    "#         pred_sum4[i] = 0\n",
    "\n",
    "# target_names = ['class 0', 'class 1']\n",
    "# print(classification_report(y_val, pred_sum4, target_names=target_names))\n",
    "\n",
    "# val_acc = metrics.accuracy_score(y_val,pred_sum4)#验证集上的auc值\n",
    "# print(val_acc)\n",
    "# print(np.sum(pred_sum4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 0 1 ... 0 3 0]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "    class 0       1.00      0.35      0.52      2867\n",
      "    class 1       0.07      0.98      0.12       133\n",
      "\n",
      "avg / total       0.96      0.38      0.50      3000\n",
      "\n",
      "0.379\n",
      "1990\n"
     ]
    }
   ],
   "source": [
    "pred_sum_sum = pred_sum + pred_sum1 + pred_sum2\n",
    "print(pred_sum_sum)\n",
    "for i in range(3000):\n",
    "    if pred_sum_sum[i] >= 1:\n",
    "        pred_sum_sum[i] = 1\n",
    "    else:\n",
    "        pred_sum_sum[i] = 0\n",
    "\n",
    "target_names = ['class 0', 'class 1']\n",
    "print(classification_report(y_val, pred_sum_sum, target_names=target_names))\n",
    "\n",
    "val_acc = metrics.accuracy_score(y_val,pred_sum_sum)#验证集上的auc值\n",
    "print(val_acc)\n",
    "print(np.sum(pred_sum_sum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "133"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
